{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 15.3 项目实施"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "WebDriverException",
     "evalue": "Message: Unable to find a matching set of capabilities\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-90b2c7d1c37a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mselenium\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdriver\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFirefox\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"https://www.dianping.com/search/category/7/10/p1\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\firefox\\webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, firefox_profile, firefox_binary, timeout, capabilities, proxy, executable_path, firefox_options, log_path)\u001b[0m\n\u001b[1;32m    150\u001b[0m                 \u001b[0mcommand_executor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mexecutor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m                 \u001b[0mdesired_capabilities\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcapabilities\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m                 keep_alive=True)\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m         \u001b[1;31m# Selenium remote\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, command_executor, desired_capabilities, browser_profile, proxy, keep_alive, file_detector)\u001b[0m\n\u001b[1;32m     96\u001b[0m             warnings.warn(\"Please use FirefoxOptions to set browser profile\",\n\u001b[1;32m     97\u001b[0m                           DeprecationWarning)\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdesired_capabilities\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbrowser_profile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_switch_to\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSwitchTo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mobile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mMobile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mstart_session\u001b[0;34m(self, capabilities, browser_profile)\u001b[0m\n\u001b[1;32m    186\u001b[0m         parameters = {\"capabilities\": w3c_caps,\n\u001b[1;32m    187\u001b[0m                       \"desiredCapabilities\": capabilities}\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mNEW_SESSION\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;34m'sessionId'\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m             \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, driver_command, params)\u001b[0m\n\u001b[1;32m    250\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 252\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    253\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[1;32m    254\u001b[0m                 response.get('value', None))\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m    192\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mexception_class\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mUnexpectedAlertPresentException\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;34m'alert'\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mWebDriverException\u001b[0m: Message: Unable to find a matching set of capabilities\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(\"https://www.dianping.com/search/category/7/10/p1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.firefox.firefox_binary import FirefoxBinary\n",
    "\n",
    "caps = webdriver.DesiredCapabilities().FIREFOX\n",
    "caps[\"marionette\"] = False\n",
    "binary = FirefoxBinary(r'D:\\Program Files\\Mozilla Firefox\\firefox.exe')\n",
    "#把上述地址改成你电脑中Firefox程序的地址\n",
    "driver = webdriver.Firefox(firefox_binary=binary, capabilities=caps)\n",
    "driver.get(\"https://www.dianping.com/search/category/7/10/p1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.firefox.firefox_binary import FirefoxBinary\n",
    "\n",
    "caps = webdriver.DesiredCapabilities().FIREFOX\n",
    "caps[\"marionette\"] = False\n",
    "\n",
    "binary = FirefoxBinary(r'D:\\Program Files\\Mozilla Firefox\\firefox.exe')\n",
    "#把上述地址改成你电脑中Firefox程序的地址\n",
    "fp = webdriver.FirefoxProfile()\n",
    "fp.set_preference(\"permissions.default.image\",2)\n",
    "fp.set_preference(\"javascript.enabled\", False)\n",
    "\n",
    "driver = webdriver.Firefox(firefox_binary=binary, firefox_profile = fp, capabilities=caps)\n",
    "driver.get(\"https://www.dianping.com/search/category/7/10/p1\")\n",
    "driver.implicitly_wait(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.firefox.firefox_binary import FirefoxBinary\n",
    "\n",
    "caps = webdriver.DesiredCapabilities().FIREFOX\n",
    "caps[\"marionette\"] = False\n",
    "\n",
    "binary = FirefoxBinary(r'D:\\Program Files\\Mozilla Firefox\\firefox.exe')\n",
    "#把上述地址改成你电脑中Firefox程序的地址\n",
    "fp = webdriver.FirefoxProfile()\n",
    "fp.set_preference(\"permissions.default.image\",2)\n",
    "fp.set_preference(\"javascript.enabled\", False)\n",
    "\n",
    "driver = webdriver.Firefox(firefox_binary=binary, firefox_profile = fp, capabilities=caps)\n",
    "driver.implicitly_wait(20)\n",
    "\n",
    "for i in range(1,51):\n",
    "    link = \"https://www.dianping.com/search/category/7/10/p\" + str(i)\n",
    "    driver.get(link)\n",
    "    soup = BeautifulSoup(driver.page_source, \"lxml\")\n",
    "    \n",
    "    output_list =[]\n",
    "    output_list = outputOneCommentResult(i, soup, output_list)\n",
    "        \n",
    "    with open('restaurant_list.csv', 'a+', newline='', encoding='utf-8') as csvfile:\n",
    "        spamwriter = csv.writer(csvfile,dialect='excel')\n",
    "        spamwriter.writerows(output_list)\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def outputOneCommentResult(page_id, soup, output_list):\n",
    "    for item in soup.find(id='shop-all-list').ul:\n",
    "        try:\n",
    "            title = item.find(class_='tit').a.text.strip()\n",
    "        except:\n",
    "            title = ''\n",
    "        try:\n",
    "            link = \"http://www.dianping.com\" + item.find(class_='tit').a['href']\n",
    "        except:\n",
    "            link = ''\n",
    "        try:\n",
    "            star = item.find(class_='comment').span['title']\n",
    "        except:\n",
    "            star = ''\n",
    "        try:\n",
    "            comment_link = \"http://www.dianping.com\" + item.find(class_='review-num')['href']\n",
    "        except:\n",
    "            comment_link = ''\n",
    "        try:\n",
    "            comment = item.find(class_='review-num').b.text.strip()\n",
    "        except:\n",
    "            comment = ''\n",
    "        try:\n",
    "            price = item.find(class_='mean-price').b.text.strip()\n",
    "        except:\n",
    "            price = ''\n",
    "        try:\n",
    "            tag = item.find(class_='tag').text.strip()\n",
    "        except:\n",
    "            tag = ''\n",
    "        try:\n",
    "            addr = item.find(class_='addr').text.strip()\n",
    "        except:\n",
    "            addr = ''\n",
    "        try:\n",
    "            commentlist = item.find(class_='comment-list').text.strip()\n",
    "            commentlist = commentlist.replace(\"\\n\",\"|\")\n",
    "        except:\n",
    "            commentlist = ''\n",
    "\n",
    "        if title != \"\":\n",
    "            output_list.append([str(page_id), title, link, star ,comment_link, comment, price, tag, addr, commentlist])\n",
    "    return output_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.3.2 获取餐厅详细信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "PermissionError",
     "evalue": "[Errno 13] Permission denied: 'restuarant_detail.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPermissionError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-850b29fdf250>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0moutput_list\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlink\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'|'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdishes_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'|'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcomment_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m     \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'restuarant_detail.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'a+'\u001b[0m \u001b[1;33m,\u001b[0m \u001b[0mnewline\u001b[0m \u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m \u001b[1;33m,\u001b[0m\u001b[0mencoding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mcsv_file\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     54\u001b[0m         \u001b[0mwriter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcsv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcsv_file\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdialect\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'excel'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0mwriter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriterows\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mPermissionError\u001b[0m: [Errno 13] Permission denied: 'restuarant_detail.csv'"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.firefox.firefox_binary import FirefoxBinary\n",
    "from selenium.webdriver.support.wait import WebDriverWait \n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "\n",
    "caps = webdriver.DesiredCapabilities().FIREFOX\n",
    "caps[\"marionette\"] = False\n",
    "\n",
    "binary = FirefoxBinary(r'D:\\Program Files\\Mozilla Firefox\\firefox.exe')\n",
    "#把上述地址改成你电脑中Firefox程序的地址\n",
    "fp = webdriver.FirefoxProfile()\n",
    "fp.set_preference(\"permissions.default.image\",2)\n",
    "fp.set_preference(\"javascript.enabled\", False)\n",
    "\n",
    "driver = webdriver.Firefox(firefox_binary=binary, firefox_profile = fp, capabilities=caps)\n",
    "driver.implicitly_wait(30)\n",
    "\n",
    "with open('restaurant_list.csv', encoding = 'utf-8') as f:\n",
    "    csv_file = csv.reader(f)\n",
    "    link_list = [[row[1],row[2]] for row in csv_file]\n",
    "\n",
    "for eachone in link_list:\n",
    "    name = eachone[0]\n",
    "    link = eachone[1]\n",
    "    output_list = []\n",
    "    driver.get(link)\n",
    "    locator = (By.CLASS_NAME, 'content')\n",
    "    WebDriverWait(driver, 20, 0.5).until(EC.presence_of_element_located(locator))\n",
    "    \n",
    "    soup = BeautifulSoup(driver.page_source, \"lxml\")\n",
    "    \n",
    "    dishes_list = []\n",
    "    for eachone in soup.find(class_=\"recommend-name\"):\n",
    "        try:\n",
    "            dishes = eachone['title'].strip() + eachone.em.text\n",
    "            dishes_list.append(dishes)\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "    comment_list = []\n",
    "    for eachone in soup.find(class_=\"content\"):\n",
    "        try:\n",
    "            comment_tag = eachone.a.text\n",
    "            comment_list.append(comment_tag)\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "    output_list.append([name, link,'|'.join(dishes_list), '|'.join(comment_list)])\n",
    "    with open('restuarant_detail.csv', 'a+' , newline ='' ,encoding = 'utf-8') as csv_file:\n",
    "        writer = csv.writer(csv_file, dialect = 'excel')\n",
    "        writer.writerows(output_list)\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
